{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/Iallen520/lhy_DL_Hw/blob/master/hw4_RNN.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "r67y9UpchZ38"
   },
   "source": [
    "# Recurrent Neural Networks\n",
    "\n",
    "本次作業是要讓同學接觸NLP當中一個簡單的task——句子分類(文本分類)\n",
    "\n",
    "給定一個句子，判斷他有沒有惡意(負面標1，正面標0)\n",
    "\n",
    "若有任何問題，歡迎來信至助教信箱ntu-ml-2020spring-ta@googlegroups.com"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9ajS_WskRo0S"
   },
   "outputs": [],
   "source": [
    "path_prefix = './'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "9YrAlczfM_w6"
   },
   "source": [
    "### Download Dataset\n",
    "有三個檔案，分別是training_label.txt、training_nolabel.txt、testing_data.txt\n",
    "\n",
    "training_label.txt：有label的training data(句子配上0 or 1)\n",
    "\n",
    "training_nolabel.txt：沒有label的training data(只有句子)，用來做semi-supervise learning\n",
    "\n",
    "testing_data.txt：你要判斷testing data裡面的句子是0 or 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8hDIokoP6464"
   },
   "outputs": [],
   "source": [
    "# this is for filtering the warnings\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "fc143hSvNGr6"
   },
   "source": [
    "### Utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ICDIhhgCY2-M"
   },
   "outputs": [],
   "source": [
    "# utils.py\n",
    "# 這個block用來先定義一些等等常用到的函式\n",
    "import torch\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "def load_training_data(path='training_label.txt'):\n",
    "    # 把training時需要的data讀進來\n",
    "    # 如果是'training_label.txt'，需要讀取label，如果是'training_nolabel.txt'，不需要讀取label\n",
    "    if 'training_label' in path:\n",
    "        with open(path, 'r', encoding='utf-8') as f:\n",
    "            lines = f.readlines()\n",
    "            lines = [line.strip('\\n').split(' ') for line in lines]\n",
    "        x = [line[2:] for line in lines]\n",
    "        y = [line[0] for line in lines]\n",
    "        return x, y\n",
    "    else:\n",
    "        with open(path, 'r', encoding='utf-8') as f:\n",
    "            lines = f.readlines()\n",
    "            x = [line.strip('\\n').split(' ') for line in lines]\n",
    "        return x\n",
    "\n",
    "def load_testing_data(path='testing_data'):\n",
    "    # 把testing時需要的data讀進來\n",
    "    with open(path, 'r', encoding='utf-8') as f:\n",
    "        lines = f.readlines()\n",
    "        X = [\"\".join(line.strip('\\n').split(\",\")[1:]).strip() for line in lines[1:]]\n",
    "        X = [sen.split(' ') for sen in X]\n",
    "    return X\n",
    "\n",
    "def evaluation(outputs, labels):\n",
    "    #outputs => probability (float)\n",
    "    #labels => labels\n",
    "    outputs[outputs>=0.5] = 1 # 大於等於0.5為有惡意\n",
    "    outputs[outputs<0.5] = 0 # 小於0.5為無惡意\n",
    "    correct = torch.sum(torch.eq(outputs, labels)).item()\n",
    "    return correct"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "oYE8UYQsNIxM"
   },
   "source": [
    "### Train Word to Vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 122
    },
    "colab_type": "code",
    "id": "cgGWaF8_2S3q",
    "outputId": "d25efb6b-de4f-43f2-8a98-e0aaa47f704e",
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "loading training data ...\nloading testing data ...\nsaving model ...\n"
    }
   ],
   "source": [
    "# w2v.py\n",
    "# 這個block是用來訓練word to vector 的 word embedding\n",
    "# 注意！這個block在訓練word to vector時是用cpu，可能要花到10分鐘以上\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import argparse\n",
    "from gensim.models import word2vec\n",
    "\n",
    "def train_word2vec(x):\n",
    "    # 訓練word to vector 的 word embedding\n",
    "    model = word2vec.Word2Vec(x, size=250, window=5, min_count=5, workers=12, iter=10, sg=1)\n",
    "    return model\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    print(\"loading training data ...\")\n",
    "    train_x, y = load_training_data('training_label.txt')\n",
    "    train_x_no_label = load_training_data('training_nolabel.txt')\n",
    "\n",
    "    print(\"loading testing data ...\")\n",
    "    test_x = load_testing_data('testing_data.txt')\n",
    "\n",
    "    model = train_word2vec(train_x + train_x_no_label + test_x)\n",
    "    \n",
    "    print(\"saving model ...\")\n",
    "    model.save(os.path.join(path_prefix, 'w2v_all.model'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "3wHLtS0wNR6w"
   },
   "source": [
    "### Data Preprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CfGKiOitk5ob"
   },
   "outputs": [],
   "source": [
    "# preprocess.py\n",
    "# 這個block用來做data的預處理\n",
    "from torch import nn\n",
    "from gensim.models import Word2Vec\n",
    "\n",
    "class Preprocess():\n",
    "    def __init__(self, sentences, sen_len, w2v_path=\"./w2v.model\"):\n",
    "        self.w2v_path = w2v_path\n",
    "        self.sentences = sentences\n",
    "        self.sen_len = sen_len\n",
    "        self.idx2word = []\n",
    "        self.word2idx = {}\n",
    "        self.embedding_matrix = []\n",
    "        \n",
    "    def get_w2v_model(self):\n",
    "        # 把之前訓練好的word to vec 模型讀進來\n",
    "        self.embedding = Word2Vec.load(self.w2v_path)\n",
    "        self.embedding_dim = self.embedding.vector_size\n",
    "        \n",
    "    def add_embedding(self, word):\n",
    "        # 把word加進embedding，並賦予他一個隨機生成的representation vector\n",
    "        # word只會是\"<PAD>\"或\"<UNK>\"\n",
    "        vector = torch.empty(1, self.embedding_dim)\n",
    "        torch.nn.init.uniform_(vector)\n",
    "        self.word2idx[word] = len(self.word2idx)\n",
    "        self.idx2word.append(word)\n",
    "        self.embedding_matrix = torch.cat([self.embedding_matrix, vector], 0)\n",
    "        \n",
    "    def make_embedding(self, load=True):\n",
    "        print(\"Get embedding ...\")\n",
    "        # 取得訓練好的 Word2vec word embedding\n",
    "        if load:\n",
    "            print(\"loading word to vec model ...\")\n",
    "            self.get_w2v_model()\n",
    "        else:\n",
    "            raise NotImplementedError\n",
    "        # 製作一個 word2idx 的 dictionary\n",
    "        # 製作一個 idx2word 的 list\n",
    "        # 製作一個 word2vector 的 list\n",
    "        for i, word in enumerate(self.embedding.wv.vocab):\n",
    "            print('get words #{}'.format(i+1), end='\\r')\n",
    "            #e.g. self.word2index['魯'] = 1 \n",
    "            #e.g. self.index2word[1] = '魯'\n",
    "            #e.g. self.vectors[1] = '魯' vector\n",
    "            self.word2idx[word] = len(self.word2idx)\n",
    "            self.idx2word.append(word)\n",
    "            self.embedding_matrix.append(self.embedding[word])\n",
    "        self.embedding_matrix = torch.tensor(self.embedding_matrix)\n",
    "        # 將\"<PAD>\"跟\"<UNK>\"加進embedding裡面\n",
    "        self.add_embedding(\"<PAD>\")\n",
    "        self.add_embedding(\"<UNK>\")\n",
    "        print(\"total words: {}\".format(len(self.embedding_matrix)))\n",
    "        return self.embedding_matrix\n",
    "    \n",
    "    def pad_sequence(self, sentence):\n",
    "        # 將每個句子變成一樣的長度\n",
    "        if len(sentence) > self.sen_len:\n",
    "            sentence = sentence[:self.sen_len]\n",
    "        else:\n",
    "            pad_len = self.sen_len - len(sentence)\n",
    "            for _ in range(pad_len):\n",
    "                sentence.append(self.word2idx[\"<PAD>\"])\n",
    "        assert len(sentence) == self.sen_len\n",
    "        return sentence\n",
    "    \n",
    "    def sentence_word2idx(self):\n",
    "        # 把句子裡面的字轉成相對應的index\n",
    "        sentence_list = []\n",
    "        for i, sen in enumerate(self.sentences):\n",
    "            print('sentence count #{}'.format(i+1), end='\\r')\n",
    "            sentence_idx = []\n",
    "            for word in sen:\n",
    "                if word in self.word2idx.keys():\n",
    "                    sentence_idx.append(self.word2idx[word])\n",
    "                else:\n",
    "                    sentence_idx.append(self.word2idx[\"<UNK>\"])\n",
    "            # 將每個句子變成一樣的長度\n",
    "            sentence_idx = self.pad_sequence(sentence_idx)\n",
    "            sentence_list.append(sentence_idx)\n",
    "        return torch.LongTensor(sentence_list)\n",
    "    \n",
    "    def labels_to_tensor(self, y):\n",
    "        # 把labels轉成tensor\n",
    "        y = [int(label) for label in y]\n",
    "        return torch.LongTensor(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "3WJB7go5NWL0"
   },
   "source": [
    "### Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "XketwKs4lFfB"
   },
   "outputs": [],
   "source": [
    "# data.py\n",
    "# 實作了dataset所需要的'__init__', '__getitem__', '__len__'\n",
    "# 好讓dataloader能使用\n",
    "import torch\n",
    "from torch.utils import data\n",
    "\n",
    "class TwitterDataset(data.Dataset):\n",
    "    \"\"\"\n",
    "    Expected data shape like:(data_num, data_len)\n",
    "    Data can be a list of numpy array or a list of lists\n",
    "    input data shape : (data_num, seq_len, feature_dim)\n",
    "    \"\"\"\n",
    "    def __init__(self, X, y):\n",
    "        self.data = X\n",
    "        self.label = y\n",
    "        \n",
    "    def __getitem__(self, idx):\n",
    "        if self.label is None: return self.data[idx]\n",
    "        return self.data[idx], self.label[idx]\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "uNJ8xWIMNa2r"
   },
   "source": [
    "### Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZS6RJADulIq1"
   },
   "outputs": [],
   "source": [
    "# model.py\n",
    "# 這個block是要拿來訓練的模型\n",
    "import torch\n",
    "from torch import nn\n",
    "\n",
    "class LSTM_Net(nn.Module):\n",
    "    def __init__(self, embedding, embedding_dim, hidden_dim, num_layers, dropout=0.5, fix_embedding=True):\n",
    "        super(LSTM_Net, self).__init__()\n",
    "        # 製作 embedding layer\n",
    "        self.embedding = torch.nn.Embedding(embedding.size(0), embedding.size(1))\n",
    "        self.embedding.weight = torch.nn.Parameter(embedding)\n",
    "        # 是否將 embedding fix住，如果fix_embedding為False，在訓練過程中，embedding也會跟著被訓練\n",
    "        self.embedding.weight.requires_grad = False if fix_embedding else True\n",
    "        self.embedding_dim = embedding.size(1)\n",
    "        self.hidden_dim = hidden_dim\n",
    "        self.num_layers = num_layers\n",
    "        self.dropout = dropout\n",
    "        self.lstm = nn.LSTM(embedding_dim, hidden_dim, num_layers=num_layers, batch_first=True)\n",
    "        self.classifier = nn.Sequential( nn.Dropout(dropout),\n",
    "                                         nn.Linear(hidden_dim, 1),\n",
    "                                         nn.Sigmoid() )\n",
    "    def forward(self, inputs):\n",
    "        inputs = self.embedding(inputs)\n",
    "        x, _ = self.lstm(inputs, None)\n",
    "        # x 的 dimension (batch, seq_len, hidden_size)\n",
    "        # 取用 LSTM 最後一層的 hidden state\n",
    "        x = x[:, -1, :] \n",
    "        x = self.classifier(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "aWlpEL0sNc10"
   },
   "source": [
    "### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4QR4MMz-lR7i"
   },
   "outputs": [],
   "source": [
    "# train.py\n",
    "# 這個block是用來訓練模型的\n",
    "import torch\n",
    "from torch import nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "def training(batch_size, n_epoch, lr, model_dir, train, valid, model, device):\n",
    "    total = sum(p.numel() for p in model.parameters())\n",
    "    trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "    print('\\nstart training, parameter total:{}, trainable:{}\\n'.format(total, trainable))\n",
    "    model.train() # 將model的模式設為train，這樣optimizer就可以更新model的參數\n",
    "    criterion = nn.BCELoss() # 定義損失函數，這裡我們使用binary cross entropy loss\n",
    "    t_batch = len(train) \n",
    "    v_batch = len(valid) \n",
    "    optimizer = optim.Adam(model.parameters(), lr=lr) # 將模型的參數給optimizer，並給予適當的learning rate\n",
    "    total_loss, total_acc, best_acc = 0, 0, 0\n",
    "    for epoch in range(n_epoch):\n",
    "        total_loss, total_acc = 0, 0\n",
    "        # 這段做training\n",
    "        for i, (inputs, labels) in enumerate(train):\n",
    "            inputs = inputs.to(device, dtype=torch.long) # device為\"cuda\"，將inputs轉成torch.cuda.LongTensor\n",
    "            labels = labels.to(device, dtype=torch.float) # device為\"cuda\"，將labels轉成torch.cuda.FloatTensor，因為等等要餵進criterion，所以型態要是float\n",
    "            optimizer.zero_grad() # 由於loss.backward()的gradient會累加，所以每次餵完一個batch後需要歸零\n",
    "            outputs = model(inputs) # 將input餵給模型\n",
    "            outputs = outputs.squeeze() # 去掉最外面的dimension，好讓outputs可以餵進criterion()\n",
    "            loss = criterion(outputs, labels) # 計算此時模型的training loss\n",
    "            loss.backward() # 算loss的gradient\n",
    "            optimizer.step() # 更新訓練模型的參數\n",
    "            correct = evaluation(outputs, labels) # 計算此時模型的training accuracy\n",
    "            total_acc += (correct / batch_size)\n",
    "            total_loss += loss.item()\n",
    "            print('[ Epoch{}: {}/{} ] loss:{:.3f} acc:{:.3f} '.format(\n",
    "            \tepoch+1, i+1, t_batch, loss.item(), correct*100/batch_size), end='\\r')\n",
    "        print('\\nTrain | Loss:{:.5f} Acc: {:.3f}'.format(total_loss/t_batch, total_acc/t_batch*100))\n",
    "\n",
    "        # 這段做validation\n",
    "        model.eval() # 將model的模式設為eval，這樣model的參數就會固定住\n",
    "        with torch.no_grad():\n",
    "            total_loss, total_acc = 0, 0\n",
    "            for i, (inputs, labels) in enumerate(valid):\n",
    "                inputs = inputs.to(device, dtype=torch.long) # device為\"cuda\"，將inputs轉成torch.cuda.LongTensor\n",
    "                labels = labels.to(device, dtype=torch.float) # device為\"cuda\"，將labels轉成torch.cuda.FloatTensor，因為等等要餵進criterion，所以型態要是float\n",
    "                outputs = model(inputs) # 將input餵給模型\n",
    "                outputs = outputs.squeeze() # 去掉最外面的dimension，好讓outputs可以餵進criterion()\n",
    "                loss = criterion(outputs, labels) # 計算此時模型的validation loss\n",
    "                correct = evaluation(outputs, labels) # 計算此時模型的validation accuracy\n",
    "                total_acc += (correct / batch_size)\n",
    "                total_loss += loss.item()\n",
    "\n",
    "            print(\"Valid | Loss:{:.5f} Acc: {:.3f} \".format(total_loss/v_batch, total_acc/v_batch*100))\n",
    "            if total_acc > best_acc:\n",
    "                # 如果validation的結果優於之前所有的結果，就把當下的模型存下來以備之後做預測時使用\n",
    "                best_acc = total_acc\n",
    "                #torch.save(model, \"{}/val_acc_{:.3f}.model\".format(model_dir,total_acc/v_batch*100))\n",
    "                torch.save(model, \"{}/ckpt.model\".format(model_dir))\n",
    "                print('saving model with acc {:.3f}'.format(total_acc/v_batch*100))\n",
    "        print('-----------------------------------------------')\n",
    "        model.train() # 將model的模式設為train，這樣optimizer就可以更新model的參數（因為剛剛轉成eval模式）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "qF5YQrupNfCS"
   },
   "source": [
    "### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2X2wkdAYxHYA"
   },
   "outputs": [],
   "source": [
    "# test.py\n",
    "# 這個block用來對testing_data.txt做預測\n",
    "import torch\n",
    "from torch import nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "def testing(batch_size, test_loader, model, device):\n",
    "    model.eval()\n",
    "    ret_output = []\n",
    "    with torch.no_grad():\n",
    "        for i, inputs in enumerate(test_loader):\n",
    "            inputs = inputs.to(device, dtype=torch.long)\n",
    "            outputs = model(inputs)\n",
    "            outputs = outputs.squeeze()\n",
    "            outputs[outputs>=0.5] = 1 # 大於等於0.5為負面\n",
    "            outputs[outputs<0.5] = 0 # 小於0.5為正面\n",
    "            ret_output += outputs.int().tolist()\n",
    "    \n",
    "    return ret_output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "dfnKj0KXNeoz"
   },
   "source": [
    "### Main"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 578
    },
    "colab_type": "code",
    "id": "EztIWqCmlZof",
    "outputId": "961deee1-4726-445b-cd49-ab4fcc0e14cf",
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "loading data ...\nGet embedding ...\nloading word to vec model ...\nget words #55777total words: 55779\nsentence count #200000\nstart training, parameter total:14447001, trainable:502251\n\n[ Epoch1: 1485/1485 ] loss:0.430 acc:28.125 \nTrain | Loss:0.49955 Acc: 74.026\nValid | Loss:0.42663 Acc: 79.549 \nsaving model with acc 79.549\n-----------------------------------------------\n[ Epoch2: 1485/1485 ] loss:0.386 acc:31.250 \nTrain | Loss:0.41560 Acc: 81.007\nValid | Loss:0.41083 Acc: 80.340 \nsaving model with acc 80.340\n-----------------------------------------------\n[ Epoch3: 1485/1485 ] loss:0.505 acc:25.000\nTrain | Loss:0.39402 Acc: 82.147\nValid | Loss:0.40237 Acc: 80.736 \nsaving model with acc 80.736\n-----------------------------------------------\n[ Epoch4: 1485/1485 ] loss:0.334 acc:32.031 \nTrain | Loss:0.37755 Acc: 83.163\nValid | Loss:0.39568 Acc: 81.260 \nsaving model with acc 81.260\n-----------------------------------------------\n[ Epoch5: 1485/1485 ] loss:0.249 acc:33.594\nTrain | Loss:0.35934 Acc: 83.997\nValid | Loss:0.39491 Acc: 81.072 \n-----------------------------------------------\n"
    }
   ],
   "source": [
    "# main.py\n",
    "import os\n",
    "import torch\n",
    "import argparse\n",
    "import numpy as np\n",
    "from torch import nn\n",
    "from gensim.models import word2vec\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# 通過torch.cuda.is_available()的回傳值進行判斷是否有使用GPU的環境，如果有的話device就設為\"cuda\"，沒有的話就設為\"cpu\"\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# 處理好各個data的路徑\n",
    "train_with_label = os.path.join(path_prefix, 'training_label.txt')\n",
    "train_no_label = os.path.join(path_prefix, 'training_nolabel.txt')\n",
    "testing_data = os.path.join(path_prefix, 'testing_data.txt')\n",
    "\n",
    "w2v_path = os.path.join(path_prefix, 'w2v_all.model') # 處理word to vec model的路徑\n",
    "\n",
    "# 定義句子長度、要不要固定embedding、batch大小、要訓練幾個epoch、learning rate的值、model的資料夾路徑\n",
    "sen_len = 30\n",
    "fix_embedding = True # fix embedding during training\n",
    "batch_size = 128\n",
    "epoch = 5\n",
    "lr = 0.001\n",
    "model_dir = path_prefix # model directory for checkpoint model\n",
    "\n",
    "print(\"loading data ...\") # 把'training_label.txt'跟'training_nolabel.txt'讀進來\n",
    "train_x, y = load_training_data(train_with_label)\n",
    "train_x_no_label = load_training_data(train_no_label)\n",
    "\n",
    "# 對input跟labels做預處理\n",
    "preprocess = Preprocess(train_x, sen_len, w2v_path=w2v_path)\n",
    "embedding = preprocess.make_embedding(load=True)\n",
    "train_x = preprocess.sentence_word2idx()\n",
    "y = preprocess.labels_to_tensor(y)\n",
    "\n",
    "# 製作一個model的對象\n",
    "model = LSTM_Net(embedding, embedding_dim=250, hidden_dim=250, num_layers=1, dropout=0.5, fix_embedding=fix_embedding)\n",
    "model = model.to(device) # device為\"cuda\"，model使用GPU來訓練(餵進去的inputs也需要是cuda tensor)\n",
    "\n",
    "# 把data分為training data跟validation data(將一部份training data拿去當作validation data)\n",
    "X_train, X_val, y_train, y_val = train_x[:190000], train_x[190000:], y[:190000], y[190000:]\n",
    "\n",
    "# 把data做成dataset供dataloader取用\n",
    "train_dataset = TwitterDataset(X=X_train, y=y_train)\n",
    "val_dataset = TwitterDataset(X=X_val, y=y_val)\n",
    "\n",
    "# 把data 轉成 batch of tensors\n",
    "train_loader = torch.utils.data.DataLoader(dataset = train_dataset,\n",
    "                                            batch_size = batch_size,\n",
    "                                            shuffle = True,\n",
    "                                            num_workers = 8)\n",
    "\n",
    "val_loader = torch.utils.data.DataLoader(dataset = val_dataset,\n",
    "                                            batch_size = batch_size,\n",
    "                                            shuffle = False,\n",
    "                                            num_workers = 8)\n",
    "\n",
    "# 開始訓練\n",
    "training(batch_size, epoch, lr, model_dir, train_loader, val_loader, model, device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "8fQeaQNeNm3L"
   },
   "source": [
    "### Predict and Write to csv file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 241
    },
    "colab_type": "code",
    "id": "vFvjFQopxVrt",
    "outputId": "3bf30696-bf76-463b-fc7f-f0b6375a0e00",
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "loading testing data ...\nGet embedding ...\nloading word to vec model ...\nget words #55777total words: 55779\nsentence count #200000\nload model ...\nsave csv ...\nFinish Predicting\n"
    }
   ],
   "source": [
    "# 開始測試模型並做預測\n",
    "print(\"loading testing data ...\")\n",
    "test_x = load_testing_data(testing_data)\n",
    "preprocess = Preprocess(test_x, sen_len, w2v_path=w2v_path)\n",
    "embedding = preprocess.make_embedding(load=True)\n",
    "test_x = preprocess.sentence_word2idx()\n",
    "test_dataset = TwitterDataset(X=test_x, y=None)\n",
    "test_loader = torch.utils.data.DataLoader(dataset = test_dataset,\n",
    "                                            batch_size = batch_size,\n",
    "                                            shuffle = False,\n",
    "                                            num_workers = 8)\n",
    "print('\\nload model ...')\n",
    "model = torch.load(os.path.join(model_dir, 'ckpt.model'))\n",
    "outputs = testing(batch_size, test_loader, model, device)\n",
    "\n",
    "# 寫到csv檔案供上傳kaggle\n",
    "tmp = pd.DataFrame({\"id\":[str(i) for i in range(len(test_x))],\"label\":outputs})\n",
    "print(\"save csv ...\")\n",
    "tmp.to_csv(os.path.join(path_prefix, 'predict.csv'), index=False)\n",
    "print(\"Finish Predicting\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "include_colab_link": true,
   "name": "hw4_RNN.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}